\documentclass[10pt,oneside]{book}

\input{macros_orig.tex}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\pagestyle{empty}
\thispagestyle{empty}

\begin{description}
  \item{\bf Step function:} 
    $$\text{step}(z) =
    \begin{cases}
      0 & \text{if $z<0$}\\
      1 & \text{otherwise}
    \end{cases}$$
  \item{\bf Rectified linear unit (ReLU):} 
    $$\text{ReLU}(z) =
    \begin{cases}
      0 & \text{if $z<0$}\\
      z & \text{otherwise}
    \end{cases} = \max(0,z)$$ 
  \item{\bf Sigmoid function:} Also known as a {\em logistic} function, can
    be interpreted as probability, because for any value of $z$ the
    output is in $(0, 1)$
    $$\sigma(z) = \frac{1}{1+e^{-z}}$$
  \item{\bf Hyperbolic tangent:} Always in the range $(-1, 1)$
 $$\tanh(z) = \frac{e^z - e^{-z}}{e^z + e^{-z}}$$
\item{\bf Softmax function:}
Takes a whole vector $Z \in \R^n$ and generates as output a vector
$A \in (0, 1)^n$ with the property that $\sum_{i = 1}^n A_i = 1$,
which means we can interpret it as a probability distribution over $n$ items:
\[\text{softmax}(z) =
  \begin{bmatrix}
    \exp(z_1) / \sum_{i} \exp(z_i) \\
    \vdots \\
    \exp(z_n) / \sum_{i} \exp(z_i)
\end{bmatrix}\]

\end{description}

\end{document}
